package com.chance.cc.crawler.development.bootstrap.jingdong;

import com.alibaba.fastjson.JSON;
import com.chance.cc.crawler.core.downloader.HttpConfig;
import com.chance.cc.crawler.core.downloader.HttpPage;
import com.chance.cc.crawler.core.downloader.HttpRequest;
import com.chance.cc.crawler.core.downloader.http.HttpClientDownloader;
import com.chance.cc.crawler.core.downloader.http.HttpClientGenerator;
import com.chance.cc.crawler.core.downloader.proxy.Proxy;
import com.chance.cc.crawler.core.downloader.proxy.SiteConfigProxyProvider;
import com.chance.cc.crawler.core.selector.Html;
import com.chance.cc.crawler.core.selector.Json;
import com.chance.cc.crawler.core.selector.Selectable;
import com.chance.cc.crawler.development.utils.RedisReader;
import groovy.util.logging.Slf4j;
import org.apache.commons.lang3.StringUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.regex.Matcher;
import java.util.regex.Pattern;

import static com.chance.cc.crawler.core.script.crawler.CrawlerScript.$_type;

/**
 * @author bx
 * @date 2021/1/8 0008 15:37
 */
@Slf4j
public class JDH5CommentHttpTest {
    public static Logger logger = LoggerFactory.getLogger(JDH5CommentHttpTest.class);

    private static Proxy proxy = new Proxy();
    static {
        //代理配置
        proxy.setHost("http-dyn.abuyun.com");
        proxy.setPort(9020);
        proxy.setUsername("HL89Q19E86E2987D");
        proxy.setPassword("71F33D94CE5F7BF2");
    }

    public static void main(String[] args) {

        HttpConfig httpConfig = HttpConfig.me("test");
        httpConfig.setProxy(proxy);
        httpConfig.setUseProxy(true);

        HttpClientDownloader downloader = new HttpClientDownloader();
        downloader.setClientGenerator(new HttpClientGenerator());
        downloader.setProxyProvider(new SiteConfigProxyProvider());

        for (int i=1;i<=1;i++){
            new Thread(new TestGzipRun(downloader,httpConfig),"thread_"+i).start();
        }
    }

    static class TestRun implements Runnable{

        HttpClientDownloader downloader;
        HttpConfig httpConfig;

        public TestRun(HttpClientDownloader downloader, HttpConfig httpConfig) {
            this.downloader = downloader;
            this.httpConfig = httpConfig;
        }

        @Override
        public void run() {
            String url = "https://xtv.xcar.com.cn/all/cid701/";
            HttpRequest httpRequest = new HttpRequest();

            for(int i=0;i<1;i++){
                httpRequest.setUrl(String.format(url,Math.random()));
                HttpPage httpPage = downloader.download(httpRequest, httpConfig);
                System.out.println(httpPage);
                Html responseHtml = httpPage.getHtml();
                List<Selectable> nodes = responseHtml.xpath("//div[@class=\"list_mod\"]//dl").nodes();
                for (Selectable node : nodes) {

                    String articleUrl = node.xpath(".//a/@href").get();
                    String releaseTime = node.xpath(".//span[@class=\"time\"]").get();


                    System.out.println(releaseTime);
                }

                RedisReader redisReader = new RedisReader("192.168.1.215",6379,3);
                redisReader.del( "list","map");
//                System.out.println(s);
            }
        }
    }

    static class TestArticleRun implements Runnable{

        HttpClientDownloader downloader;
        HttpConfig httpConfig;

        public TestArticleRun(HttpClientDownloader downloader, HttpConfig httpConfig) {
            this.downloader = downloader;
            this.httpConfig = httpConfig;
        }

        @Override
        public void run() {
            String url = "https://info.xcar.com.cn/202101/news_2055414_1.html?viewtype=all";
            HttpRequest httpRequest = new HttpRequest();

            for(int i=0;i<1;i++){
                httpRequest.setUrl(String.format(url,Math.random()));
                HttpPage httpPage = downloader.download(httpRequest, httpConfig);

                Matcher mtAuthor = Pattern.compile("data: \\{([^}])*\\}").matcher(httpPage.getRawText());
                mtAuthor.find();
                String data = mtAuthor.group();
                System.out.println(data.substring(data.indexOf("data: ")+"data: ".length()));
            }
        }
    }

    static class TestKoubeiRun implements Runnable{

        HttpClientDownloader downloader;
        HttpConfig httpConfig;

        public TestKoubeiRun(HttpClientDownloader downloader, HttpConfig httpConfig) {
            this.downloader = downloader;
            this.httpConfig = httpConfig;
        }

        @Override
        public void run() {
            String url = "https://newcar.xcar.com.cn/auto/index.php?r=reputation/reputation/GetAjaxKbList3&page=1&pserid=8&jh=0&wd=0";
            HttpRequest httpRequest = new HttpRequest();
            httpRequest.addHeader("user-agent","Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/87.0.4280.141 Safari/537.36");
            httpRequest.addHeader("cookie","nguv=c_160999737235228227110117218341611798; _Xdwuv=5ff69c3ce5e31; _Xdwnewuv=1; _PVXuv=5ff69c3df8614; __jsluid_s=49e976754d92ae5b3a1f9be9e2f407db; __jsluid_h=ff270f9a2198fb6530f6e9e06bae425a; __clickidc=161000895164367053; fw_slc=1%3A1610601454%3B1%3A1610601458%3B1%3A1610601459%3B1%3A1610601461%3B1%3A1610601462; fw_pvc=1%3A1610601300%3B1%3A1610601458%3B1%3A1610601461%3B1%3A1610601667%3B1%3A1610601941; fw_clc=1%3A1610601456%3B1%3A1610601460%3B1%3A1610601666%3B1%3A1610601940%3B1%3A1610601992; fw_exc=1%3A1610601456%3B1%3A1610601457%3B1%3A1610601671%3B1%3A1610603232%3B1%3A1610603244; Hm_lvt_53eb54d089f7b5dd4ae2927686b183e0=1610247261,1610333953,1610601302,1610676357; place_prid_lin=2; place_crid_lin=507; place_prname=%E4%B8%8A%E6%B5%B7%E5%B8%82; iwt_uuid=55c322d6-dcca-4fb4-8174-19f8fadc38eb; _locationInfo_=%7Burl%3A%22http%3A%2F%2Fsh.xcar.com.cn%2F%22%2Ccity_id%3A%22507%22%2Cprovince_id%3A%222%22%2C%20city_name%3A%22%25E4%25B8%258A%25E6%25B5%25B7%22%7D; isRemoveCookie=1; place_crname=%E4%B8%8A%E6%B5%B7%E8%BD%A6%E5%B8%82; uv_firstv_refers=https%3A//xtv.xcar.com.cn/show/202101/id_228119.html; __jsl_clearance_s=1610696223.082|0|olPDY0Wa3k3aAbSDhBWQ8W%2BY8UE%3D; zg_did=%7B%22did%22%3A%20%22176db52511822c-0749a9c4fbd349-c791039-100200-176db525119196%22%7D; zg_8f3d0255011c4bc5bae66beca6584825=%7B%22sid%22%3A%201610696223475%2C%22updated%22%3A%201610696260422%2C%22info%22%3A%201610603234040%2C%22superProperty%22%3A%20%22%7B%5C%22platform_type%5C%22%3A%20%5C%22PC%5C%22%2C%5C%22login_id%5C%22%3A%20null%2C%5C%22project_name%5C%22%3A%20%5C%22XCAR%5C%22%2C%5C%22login_status%5C%22%3A%200%7D%22%2C%22platform%22%3A%20%22%7B%7D%22%2C%22utm%22%3A%20%22%7B%7D%22%2C%22referrerDomain%22%3A%20%22www.xcar.com.cn%22%2C%22landHref%22%3A%20%22https%3A%2F%2Fxtv.xcar.com.cn%2F%22%2C%22zs%22%3A%200%2C%22sc%22%3A%200%2C%22firstScreen%22%3A%201610696223475%7D; _Xdwstime=1610696261; Hm_lpvt_53eb54d089f7b5dd4ae2927686b183e0=1610696262");

            for(int i=0;i<100;i++){
                List<Map<String,String>> crawlerDataList = new ArrayList<>();

                httpRequest.setUrl(String.format(url,Math.random()));
                HttpPage httpPage = downloader.download(httpRequest, httpConfig);
                Html responseHtml = httpPage.getHtml();
                List<Selectable> nodes = responseHtml.xpath("//div[@class=\"home_list clearfix\"]").nodes();
                for(Selectable node : nodes){
                    Map<String,String> crawlerDataMap = new HashMap<>();
                    listInfos(node,crawlerDataMap);
                    userScoreInfos(node,crawlerDataMap);
                    otherInfos(node,crawlerDataMap);
                    crawlerDataList.add(crawlerDataMap);
                }
                System.out.println(httpPage.getStatusCode() + "-->" + JSON.toJSONString(crawlerDataList));

            }

        }

        private void listInfos(Selectable node,Map<String,String> crawlerDataMap){
            List<Selectable> listInforNodes = node.xpath(".//div[@class=\"list_infor\"]/dl[@class=\"clearfix\"]").nodes();
            for (Selectable listInforNode : listInforNodes) {
                String desc = listInforNode.xpath("./dt").get();
                switch (desc){
                    case "爱车车型":
                        crawlerDataMap.put("model_url","https://newcar.xcar.com.cn"+listInforNode.xpath(".//a/@href").get());
                        crawlerDataMap.put("model",listInforNode.xpath(".//a/text()").get());
                    case "购入时间":
                        crawlerDataMap.put("time_buy",listInforNode.xpath("./dd/text()").get());
                    case "购车地点":
                        crawlerDataMap.put("vendor",listInforNode.xpath("./dd/text()").get());
                    case "裸车价格":
                        crawlerDataMap.put("buy_price",listInforNode.xpath("./dd/text()").get());
                    case "当前油耗":
                        crawlerDataMap.put("fuel_economy",listInforNode.xpath("./dd/text()").get());
                }
            }
        }

        private void userScoreInfos(Selectable node,Map<String,String> crawlerDataMap){
            String userName = node.xpath(".//div[@class=\"name_lf\"]/dl[@class=\"clearfix\"]//span/text()").get();
            crawlerDataMap.put("author",userName);

            String scoreTit = node.xpath(".//div[@class=\"name_lf\"]//div[@class=\"score_tit\"]/text()").get()+
                    node.xpath(".//div[@class=\"name_lf\"]//div[@class=\"score_tit\"]/em/text()").get();
            StringBuffer scoreBuffer = new StringBuffer(scoreTit);
            List<Selectable> scoreNodeList = node.xpath(".//div[@class=\"name_lf\"]//ul[@class=\"score_list\"]/li").nodes();
            for (Selectable scoreNodeInfo : scoreNodeList) {
                String score = scoreNodeInfo.xpath("./span/text()").get() + scoreNodeInfo.xpath("./em/text()").get();
                scoreBuffer.append("\n").append(score);
            }
            crawlerDataMap.put("values",scoreBuffer.toString());
        }

        private void otherInfos(Selectable node,Map<String,String> crawlerDataMap){
            List<String> targets = node.xpath(".//div[@class=\"purpose clearfix\"]/em/text()").all();
            crawlerDataMap.put("target", StringUtils.joinWith(",",targets.toArray()));

            List<String> images = node.xpath(".//ul[@class=\"photo_ul\"]//a/img/@src").all();
            crawlerDataMap.put("imageurl",StringUtils.joinWith("\\x01",images.toArray()));

            List<String> content = node.xpath(".//div[@class=\"review_post\"]/dl/*/text()").all();
            crawlerDataMap.put("content",StringUtils.joinWith("",content.toArray()));

            String like = node.xpath(".//a[@class=\"zan_ment\"]/text()").get();
            crawlerDataMap.put("like",like);

            String releaseTime = node.xpath(".//div[@class=\"publish\"]/text()").get();
            crawlerDataMap.put("releaseTime",releaseTime);

            List<String> spans = node.xpath(".//div[@class=\"publish\"]/span/@class").all();
            String isBest=null;
            for (String span : spans) {
                if (span.startsWith("ssen_")){
                    isBest = span;
                }
            }
            crawlerDataMap.put("isBest",isBest);
        }
    }

    static class TestForumRun implements Runnable{

        HttpClientDownloader downloader;
        HttpConfig httpConfig;

        public TestForumRun(HttpClientDownloader downloader, HttpConfig httpConfig) {
            this.downloader = downloader;
            this.httpConfig = httpConfig;
        }

        @Override
        public void run() {
            String url = "https://www.xcar.com.cn/bbs/viewthread.php?tid=27030255&page=3470";
            HttpRequest httpRequest = new HttpRequest();

            for(int i=0;i<100;i++){
                List<Map<String,String>> crawlerDataList = new ArrayList<>();

                httpRequest.setUrl(String.format(url,Math.random()));
                HttpPage httpPage = downloader.download(httpRequest, httpConfig);
                Html responseHtml = httpPage.getHtml();

                String dataInfo = articleDataInfo(httpPage);
                Json dataInfoJson = new Json(dataInfo);
                String author_name = dataInfoJson.jsonPath($_type + ".author_name").get();
                String author_id = dataInfoJson.jsonPath($_type + ".author_id").get();
                String content_title = dataInfoJson.jsonPath($_type + ".content_title").get();
                String content_id = dataInfoJson.jsonPath($_type + ".content_id").get();



                List<Selectable> nodes = responseHtml.xpath("//div[@class=\"details\"]/div[@class=\"floor_div\"]").nodes();

                for(Selectable node : nodes){

                }
                System.out.println(httpPage.getStatusCode() + "-->" + JSON.toJSONString(crawlerDataList));
            }
        }

        private String articleDataInfo(HttpPage page){
            String dataStr = "data: ";
            String dataInfo = null;
            Matcher mtAuthor = Pattern.compile(dataStr + "\\{([^}])*\\}").matcher(page.getRawText());
            if (mtAuthor.find()){
                dataInfo = mtAuthor.group();
                dataInfo = dataInfo.substring(dataInfo.indexOf(dataStr)+dataStr.length());

            }
            return dataInfo;
        }
    }

    static class TestGzipRun implements Runnable{

        HttpClientDownloader downloader;
        HttpConfig httpConfig;

        public TestGzipRun(HttpClientDownloader downloader, HttpConfig httpConfig) {
            this.downloader = downloader;
            this.httpConfig = httpConfig;
        }

        @Override
        public void run() {
            String url = "https://wq.jd.com/commodity/comment/getcommentlist?callback=skuJDEvalA&version=v2" +
                    "&pagesize=10&sceneval=2&score=0&sku=70256465741&sorttype=6&page=1&t=0.4184708849938641";
            HttpRequest httpRequest = new HttpRequest();

            for(int i=0;i<100;i++){
                List<Map<String,String>> crawlerDataList = new ArrayList<>();

                httpRequest.setUrl(url);
                httpRequest.addHeader("referer","https://item.jd.com/70256465741.html");
                httpRequest.addHeader("accept","*/*");
                httpRequest.addHeader("user-agent","Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/88.0.4324.104 Safari/537.36");
                HttpPage httpPage = downloader.download(httpRequest, httpConfig);

//                String requestStr = "{\"binaryContent\":false,\"cookies\":{},\"extras\":{\"productId\":\"70256465741\",\"currentPageNum\":\"1\"},\"headers\":{\"referer\":\"https://item.jd.com/70256465741.html\",\"Accept\":\"*/*\",\"user-agent\":\"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/87.0.4280.88 Safari/537.36\"},\"url\":\"https://wq.jd.com/commodity/comment/getcommentlist?callback=skuJDEvalA&version=v2&pagesize=10&sceneval=2&score=0&sku=70256465741&sorttype=6&page=1&t=0.03291762282112187\"}";
//                String requestHttpConfig = "{\"circularRedirectsAllowed\":false,\"connectionRequestTimeout\":-1,\"connectionTimeout\":30000,\"disableCookie\":false,\"httpDomain\":\"jingdong\",\"httpSite\":\"default-site\",\"proxy\":{\"host\":\"http-dyn.abuyun.com\",\"password\":\"71F33D94CE5F7BF2\",\"port\":9020,\"username\":\"HL89Q19E86E2987D\"},\"requestRetryTimes\":0,\"responseTextGenerateHtml\":true,\"socketTimeout\":20000,\"useGzip\":true,\"useProxy\":true,\"userAgent\":\"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/85.0.4183.102 Safari/537.36\"}";
//
//                HttpRequest request = JSON.parseObject(requestStr,HttpRequest.class);
//                HttpConfig httpConfig = JSON.parseObject(requestHttpConfig,HttpConfig.class);
//                HttpPage httpPage = downloader.download(request, httpConfig);

                System.out.println(JSON.toJSONString(httpRequest));
                System.out.println(JSON.toJSONString(httpConfig));
                System.out.println(httpPage.getStatusCode() + "-->" + JSON.toJSONString(httpPage.getRawText()));
            }
        }

        private String articleDataInfo(HttpPage page){
            String dataStr = "data: ";
            String dataInfo = null;
            Matcher mtAuthor = Pattern.compile(dataStr + "\\{([^}])*\\}").matcher(page.getRawText());
            if (mtAuthor.find()){
                dataInfo = mtAuthor.group();
                dataInfo = dataInfo.substring(dataInfo.indexOf(dataStr)+dataStr.length());

            }
            return dataInfo;
        }
    }
}
